#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
import json

from itemadapter import ItemAdapter

# 方法一：常规写入json文件
# class DoubanmoviesPipeline:
#     def __init__(self):
#         self.fp = open("movies_info.json", "w", encoding='utf-8')
#
#     def open_spider(self, spider):
#         print("这是爬虫开始……")
#
#     def process_item(self, item, spider):
#         item_json = json.dumps(dict(item), ensure_ascii=False)
#         self.fp.write(item_json+'\n')
#         return item
#
#     def close_spider(self, spider):
#         self.fp.close()
#         print("爬虫运行结束了……")


# 方法二：使用JsonItemExporter函数写入，但写入不自动换行
# from scrapy.exporters import JsonItemExporter
#
#
# class DoubanmoviesPipeline:
#     def __init__(self):
#         self.fp = open("movies_info1.json", "wb")
#         self.exporter = JsonItemExporter(self.fp, encoding='utf-8', ensure_ascii=False)
#         self.exporter.start_exporting()
#
#     def open_spider(self, spider):
#         print("这是爬虫开始……")
#
#     def process_item(self, item, spider):
#         self.exporter.export_item(item)
#         return item
#
#     def close_spider(self, spider):
#         self.exporter.finish_exporting()
#         self.fp.close()
#         print("爬虫运行结束了……")


# 使用JsonLinesItemExporter函数写入，写入文件可以自动换行，相对简单方便
from scrapy.exporters import JsonLinesItemExporter


class DoubanmoviesPipeline:
    def __init__(self):
        self.fp = open("movies_all.json", "wb")
        self.exporter = JsonLinesItemExporter(self.fp, encoding='utf-8', ensure_ascii=False)

    def open_spider(self, spider):
        print("这是爬虫开始……")

    def process_item(self, item, spider):
        self.exporter.export_item(item)
        return item

    def close_spider(self, spider):
        self.fp.close()
        print("爬虫运行结束了……")
